package com.eascs.spider.ant.worker.spider;

import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.eascs.spider.ant.common.proj.Platform;
import com.eascs.spider.ant.common.proj.Task;
import com.eascs.spider.ant.common.util.CommonUtil;
import com.eascs.spider.ant.common.util.Downloader;

/**
 * 新农商供应信息
 * @author hzp
 *
 */
public class NcSupplySpider extends AnotherSpider{
	
	public NcSupplySpider(Task task) {
		super(task);
	}

	@Override
	public void run() {
		try {
			if(task.getHomepage().equals(Task.HOMEPAGE_YES)){
				crawlerHomepage();//主页
			}else if(task.getUrl().contains("http://nc.mofcom.gov.cn/supply")){
				crawlerShopInfo();//店铺信息
			}else{
				crawlerSupply();//分类
			}
		} catch (Exception e) {
			logger.error("爬取发生异常,task:{}" , JSON.toJSONString(task) ,e);
			manager.addSendQueue(task);
		}
	}
	
	
	//=====================================主页部分开始==========================================
	//爬主页信息
	private void crawlerHomepage(){
		String html = downloader.setHttpMethod(Task.HTTP_GET)
								.setUrl(task.getUrl())
								.download()
								.getHtml();
		extractDirUrl(html);
	}
	
	//首页提取各分类目录url，其他不取
	private void extractDirUrl(String html){
		Matcher matcher = Pattern.compile("<a href=\"/channel/gxdj/gyxx/gyList.shtml\\?craft_index=(.*?)&amp;craftName=(.*?)\">.*?</a>").matcher(html);
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://nc.mofcom.gov.cn/channel/gxdj/gyxx/index2015.shtml");
		header.put("Host", "nc.mofcom.gov.cn");
		String url = "http://nc.mofcom.gov.cn/channel/gxdj/gyxx/gyList.shtml";
		while(matcher.find()){
			Map<String, String> param = new HashMap<>();
			param.put("craft_index", matcher.group(1));
			param.put("craftName", matcher.group(2));
			Task task = new Task();
			task.setUrl(url)
				.setMethod(Task.HTTP_GET)
				.setHomepage(Task.HOMEPAGE_NO)
				.setHeaderMap(header)
				.setParamMap(param);
			manager.addSendQueue(task);
		}
	}
	//=====================================主页部分结束==========================================
	
	//=====================================非主页部分开始==========================================
	//爬取非主页任务
	private void crawlerSupply(){
		Map<String, String> header = task.getHeaderMap();
		Map<String, String> param = task.getParamMap();
		
		String html = downloader.setUrl(task.getUrl())
							    .setHeader(header)
							    .setParam(param)
							    .setHttpMethod(Task.HTTP_GET)
							    .download()
							    .getHtml();
		extractShopUrl(html);
		turnpage(html);
	}
	
	//提取店铺url
	private void extractShopUrl(String html){
		Matcher matcher = Pattern.compile("/supply/.*?.html").matcher(html);
		while(matcher.find()){
			String url = "http://nc.mofcom.gov.cn" + matcher.group();
			Map<String, String> header = new HashMap<>();
			header.put("Host", "nc.mofcom.gov.cn");
			header.put("Referer", task.getUrl());
			Task taskTemp = new Task();
			taskTemp.setUrl(url)
					.setHeaderMap(header)
					.setHomepage(Task.HOMEPAGE_NO)
					.setMethod(Task.HTTP_GET);
			manager.addSendQueue(taskTemp);
		}
	}

	//根据店铺url提取其中信息
	private void crawlerShopInfo(){
		Map<String,String> header = task.getHeaderMap();
		String html = downloader.setUrl(task.getUrl())
							    .setHttpMethod(Task.HTTP_GET)
							    .setHeader(header)
							    .download()
							    .getHtml();
		
		if(html.contains("503 Service Unavailable")){//阿布云代理问题
			manager.addSendQueue(task);
			return;
		}
			
		Map<String, Object> data = new HashMap<>();
		Document document = Jsoup.parse(html);
		String title = document.selectFirst("div.goodsTxt header.title").text();
		data.put("title", title);
		data.put("url", task.getUrl());
		//基本信息
		Elements ps = document.select("div.goodsTxt section.con p");
		for(Element p : ps){
			if(!p.text().contains("未填")){
				if(p.text().contains("浏览")){
					data.put("浏览量", p.text());
				}else{
					String[] temps = p.text().split("：");
					if(temps.length==1)
						continue;
					if(temps[0].contains("手机"))
						temps[1] = temps[1].replace(";", "");
					data.put(temps[0], temps[1]);
				}
			}
		}
		
		//详细说明
		Elements tds = document.select("section.expTable td");
		for(Element td : tds){
			if(!td.text().contains("未填")){
				if(td.text().contains("详细说明"))
					continue;
				String[] temps = td.text().split("：");
				if(temps[0].contains("手机"))
					temps[1] = temps[1].replace(";", "");
				data.put(temps[0], temps[1]);
			}
		}
		
		//详细描述
		if(document.select("article.article p").size() >0){
			String article = document.selectFirst("article.article p").text();
			data.put("详细描述", article);
		}
		
		//请求联系方式页面
		Matcher matcher = Pattern.compile("class=\"more\" href=\"/channel/shzy/gqxx/index.shtml\\?cp_id=(.*?)\"").matcher(html);
		String cp_id = null;
		if(matcher.find()){
			cp_id = matcher.group(1);
		}else
			return;
		
		String telUrl = "http://nc.mofcom.gov.cn/channel/shzy/lxff/index.shtml?cp_id=" + cp_id;
		String telHtml = Downloader.build().setUrl(telUrl).setHttpMethod("GET").download().getHtml();
		Document telDoc = Jsoup.parse(telHtml);
		Elements telPs = telDoc.select("div.lxBox p");
		for(Element telP : telPs){
			if(telP.text().contains("未填"))
				continue;
			String[] temps = telP.text().split("： ");
			if(temps[0].contains("手机"))
				temps[1] = temps[1].replace(";", "");
			data.put(temps[0], temps[1]);
		}
		data.put("crawler_time", new Date(System.currentTimeMillis()));
		logger.info("新农商网供应信息,单条数据,data:{}" , JSON.toJSONString(data));
		mongoSession.setData(data, Platform.NC_SUPPLY);
	}	
	
	private void turnpage(String html){
		Matcher matcher = Pattern.compile("var v_PageCount = (.*?);").matcher(html);
		int pages = 0;
		if(matcher.find()){
			pages = Integer.valueOf(matcher.group(1));
		}
		if(pages == 0)
			return ;
		Matcher matcherPage = Pattern.compile("/supply/.*?.html").matcher(html);//用来检查翻页源码是否有店铺url，没有即中断
		for(int i=2;i<=pages;i++){
			Map<String, String> header = downloader.getHeader();
			header.put("Referer", downloader.getUrl());
			Map<String, String> param = downloader.getParam();
			param.put("pages", i+"");
			
			String htmlTemp = downloader.setParam(param)
									    .setHeader(header)
									    .setUrl(task.getUrl())
									    .setHttpMethod(Task.HTTP_GET)
									    .download()
									    .getHtml();
			if(!matcherPage.find()){
				logger.info("提前中断翻页,当前页数:{},task:{}" ,i ,JSON.toJSONString(task));
				break;
			}
			extractShopUrl(htmlTemp);
			CommonUtil.sleep(5000);
		}
		
	}
	//=====================================非主页部分结束==========================================
	
	public static void main(String[] args) {
		Task task = new Task();
		task.setHomepage("yes");
		task.setMethod("GET");
		task.setUrl("http://nc.mofcom.gov.cn/channel/gxdj/gyxx/gyList.shtml");
	}
}
